{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 常用数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:41:00.286082Z",
     "start_time": "2021-09-02T07:40:50.020574Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch_geometric.data import Data\n",
    "from torch_geometric.utils import add_remaining_self_loops\n",
    "import pickle "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cora数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T04:11:32.186273Z",
     "start_time": "2021-07-26T04:11:31.128400Z"
    }
   },
   "outputs": [],
   "source": [
    "path = \"data/cora/\"\n",
    "cites = path + \"cora.cites\"\n",
    "content = path + \"cora.content\"\n",
    "\n",
    "# 索引字典，转换到从0开始编码\n",
    "index_dict = dict()\n",
    "# 标签字典\n",
    "label_to_index = dict()\n",
    "\n",
    "features = []\n",
    "labels = []\n",
    "edge_index = []\n",
    "one_hot_labels = []\n",
    "\n",
    "num_label = 7\n",
    "\n",
    "with open(content,\"r\") as f:\n",
    "    nodes = f.readlines()\n",
    "    for node in nodes:\n",
    "        node_info = node.split()\n",
    "        index_dict[int(node_info[0])] = len(index_dict)\n",
    "        features.append([int(i) for i in node_info[1:-1]])\n",
    "        \n",
    "        label_str = node_info[-1]\n",
    "        if(label_str not in label_to_index.keys()):\n",
    "            label_to_index[label_str] = len(label_to_index)\n",
    "        one_hot_label = [0 for i in range(num_label)]\n",
    "        one_hot_label[label_to_index[label_str]] = 1\n",
    "        labels.append(label_to_index[label_str])\n",
    "        one_hot_labels.append(one_hot_label)\n",
    "\n",
    "with open(cites,\"r\") as f:\n",
    "    edges = f.readlines()\n",
    "    for edge in edges:\n",
    "        start, end = edge.split()\n",
    "        edge_index.append([index_dict[int(start)],index_dict[int(end)]])\n",
    "        edge_index.append([index_dict[int(end)],index_dict[int(start)]])\n",
    "    \n",
    "labels = torch.LongTensor(labels)\n",
    "one_hot_labels = torch.FloatTensor(one_hot_labels)\n",
    "features = torch.FloatTensor(features)\n",
    "# 行归一化\n",
    "# features = torch.nn.functional.normalize(features, p=1, dim=1)\n",
    "\n",
    "edge_index =  torch.LongTensor(edge_index).t()\n",
    "\n",
    "# 增加自环\n",
    "edge_index, _ = add_remaining_self_loops(edge_index)\n",
    "\n",
    "data = Data(x = features, label_feature = one_hot_labels, edge_index = edge_index.contiguous(), y = labels, num_label = num_label)\n",
    "\n",
    "with open(path+\"cora.pkl\",'wb') as f:\n",
    "    pickle.dump(data, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# citeseer数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T04:12:08.267980Z",
     "start_time": "2021-07-26T04:12:04.865235Z"
    }
   },
   "outputs": [],
   "source": [
    "path = \"data/citeseer/\"\n",
    "cites = path + \"citeseer.cites\"\n",
    "content = path + \"citeseer.content\"\n",
    "\n",
    "\n",
    "# 索引字典，转换到从0开始编码\n",
    "index_dict = dict()\n",
    "# 标签字典\n",
    "label_to_index = dict()\n",
    "\n",
    "features = []\n",
    "labels = []\n",
    "edge_index = []\n",
    "one_hot_labels = []\n",
    "\n",
    "num_label = 6\n",
    "\n",
    "with open(content,\"r\") as f:\n",
    "    nodes = f.readlines()\n",
    "    for node in nodes:\n",
    "        node_info = node.split()\n",
    "        index_dict[node_info[0]] = len(index_dict)\n",
    "        features.append([int(i) for i in node_info[1:-1]])\n",
    "        \n",
    "        label_str = node_info[-1]\n",
    "        if(label_str not in label_to_index.keys()):\n",
    "            label_to_index[label_str] = len(label_to_index)\n",
    "        one_hot_label = [0 for i in range(num_label)]\n",
    "        one_hot_label[label_to_index[label_str]] = 1\n",
    "        labels.append(label_to_index[label_str])\n",
    "        one_hot_labels.append(one_hot_label)\n",
    "        \n",
    "\n",
    "with open(cites,\"r\") as f:\n",
    "    edges = f.readlines()\n",
    "    for edge in edges:\n",
    "        try:\n",
    "            start, end = edge.split()\n",
    "            edge_index.append([index_dict[start],index_dict[end]])\n",
    "            edge_index.append([index_dict[end],index_dict[start]])\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "labels = torch.LongTensor(labels)\n",
    "one_hot_labels = torch.FloatTensor(one_hot_labels)\n",
    "features = torch.FloatTensor(features)\n",
    "# 行归一化\n",
    "# features = torch.nn.functional.normalize(features, p=1, dim=1)\n",
    "\n",
    "edge_index =  torch.LongTensor(edge_index).t()\n",
    "\n",
    "# 添加自环\n",
    "# edge_index, _ = add_remaining_self_loops(edge_index)\n",
    "\n",
    "\n",
    "data = Data(x = features, label_feature = one_hot_labels, edge_index = edge_index.contiguous(), y = labels, num_label = num_label)\n",
    "data\n",
    "\n",
    "\n",
    "with open(path+\"citeseer_no_loops.pkl\",'wb') as f:\n",
    "    pickle.dump(data, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OGB数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-15T01:47:29.541768Z",
     "start_time": "2021-09-15T01:47:25.422666Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "# 以节点任务为例，还有边任务和图任务，PygEdgePropPredDataset,PygGraphPropPredDataset\n",
    "from ogb.nodeproppred import PygNodePropPredDataset, Evaluator\n",
    "import torch_geometric.transforms as T\n",
    "\n",
    "name = 'ogbn-products'\n",
    "\n",
    "# 不使用稀疏矩阵\n",
    "dataset = PygNodePropPredDataset(name)\n",
    "evaluator = Evaluator(name=name)\n",
    "split_idx = dataset.get_idx_split()\n",
    "data = dataset[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据集构成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:46:13.668808Z",
     "start_time": "2021-09-02T07:46:13.650587Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_items([('name', 'ogbn-products'), ('dir_name', 'ogbn_products'), ('original_root', 'dataset'), ('root', 'dataset/ogbn_products'), ('meta_info', num tasks                                                                1\n",
       "num classes                                                             47\n",
       "eval metric                                                            acc\n",
       "task type                                        multiclass classification\n",
       "download_name                                                     products\n",
       "version                                                                  1\n",
       "url                      http://snap.stanford.edu/ogb/data/nodeproppred...\n",
       "add_inverse_edge                                                      True\n",
       "has_node_attr                                                         True\n",
       "has_edge_attr                                                        False\n",
       "split                                                        sales_ranking\n",
       "additional node files                                                 None\n",
       "additional edge files                                                 None\n",
       "is hetero                                                            False\n",
       "binary                                                               False\n",
       "Name: ogbn-products, dtype: object), ('download_name', 'products'), ('num_tasks', 1), ('task_type', 'multiclass classification'), ('eval_metric', 'acc'), ('__num_classes__', 47), ('is_hetero', False), ('binary', False), ('transform', ToSparseTensor()), ('pre_transform', None), ('pre_filter', None), ('__indices__', None), ('data', Data(edge_index=[2, 123718280], x=[2449029, 100], y=[2449029, 1])), ('slices', {'x': tensor([      0, 2449029]), 'edge_index': tensor([        0, 123718280]), 'y': tensor([      0, 2449029])}), ('__data_list__', [Data(edge_index=[2, 123718280], x=[2449029, 100], y=[2449029, 1])])])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.__dict__.items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-03T01:10:33.518760Z",
     "start_time": "2021-09-03T01:10:33.501780Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(edge_index=[2, 123718280], x=[2449029, 100], y=[2449029, 1])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-03T01:10:34.398880Z",
     "start_time": "2021-09-03T01:10:34.392702Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(adj_t=[2449029, 2449029, nnz=123718280], x=[2449029, 100], y=[2449029, 1])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-03T01:10:45.885562Z",
     "start_time": "2021-09-03T01:10:45.877898Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SparseTensor(row=tensor([      0,       0,       0,  ..., 2449028, 2449028, 2449028]),\n",
       "             col=tensor([    384,    2412,    7554,  ..., 1787657, 1864057, 2430488]),\n",
       "             size=(2449029, 2449029), nnz=123718280, density=0.00%)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sparse.adj_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-03T01:12:57.925649Z",
     "start_time": "2021-09-03T01:12:57.446302Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 152857,   32104,   23158,  228358,  115556, 2015179,   20518,   84680,\n",
       "         117780,    9796,   74338, 1155605,   81401,  109121,  191321, 2064036,\n",
       "          34745,  403564,   44914,  112903,    7554, 1155944,   41554, 2279548,\n",
       "        1591461, 2021621,  119810,  266190,  186457,   65022, 2037792,   83455,\n",
       "         115977, 1053264,  200419,   91688,  134402,  227054,   85465, 1331894,\n",
       "          42011, 1172458,   27145,  194591,   96967,  176470, 2402273,  642128,\n",
       "        2372192, 2334394,  221486,     384,    2412,   14553,   19089,   26151,\n",
       "          29210,   35148,   36684,   40029,   43725,   44093,   54133,   57300,\n",
       "          57851,   61252,   61275,   67280,   75096,   79919,   83908,   85330,\n",
       "          87117,   89183,   95121,   97162,  102240,  116909,  117502,  120226,\n",
       "         132547,  149449,  150080,  154133,  166173,  173341,  179305,  182270,\n",
       "         185172,  191440,  198496,  200076,  200376,  223428,  227171,  229815,\n",
       "         287006,  302764,  317669,  354784,  394021,  411357,  477417,  515292,\n",
       "         547462,  561857,  575913,  595802,  686800,  736258,  750123,  811518,\n",
       "         862237,  883497, 1001402, 1012465, 1102247, 1109815, 1113367, 1165959,\n",
       "        1229555, 1242592, 1268562, 1343830, 1380143, 1391964, 1404032, 1458050,\n",
       "        1459904, 1469339, 1480626, 1510677, 1512917, 1552384, 1586895, 1607879,\n",
       "        1618304, 1622355, 1738709, 1745625, 1779301, 1780298, 1794498, 1800482,\n",
       "        1826230, 1834554, 1872711, 1893146, 1906884, 1928137, 1929666, 1934234,\n",
       "        1937842, 1979065, 1984560, 2034637, 2052443, 2077658, 2083627, 2118786,\n",
       "        2119785, 2128561, 2136265, 2155876, 2198082, 2215350, 2259911, 2267363,\n",
       "        2306440, 2306783, 2320830, 2392200, 2393721, 2406078])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.edge_index[1][data.edge_index[0]==0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "song",
   "language": "python",
   "name": "song"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
